{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "import json\n",
    "import os\n",
    "from io import StringIO\n",
    "import io\n",
    "import unicodedata\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a data directory if it does not exist\n",
    "directory = 'data'\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "    \n",
    "directory = 'data/ASAP'\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "\n",
    "#preprocessing dine via the script available at https://github.com/nusnlp/nea/tree/master/data\n",
    "#this script creates 5 fold CV data from the ASAP dataset (training_set_rel3.tsv) based on the essay IDs\n",
    "path = os.path.join(os.getcwd(),'nea/data')\n",
    "folders = ['fold_0', 'fold_1','fold_2','fold_3', 'fold_4']\n",
    "\n",
    "#Specify which ASAP sets to use from 1-8\n",
    "sets = [1,2]\n",
    "#f_path = [os.path.join(path,f) for f in folders]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(t_):\n",
    "    t_ = re.sub('\\s+',' ',t_)\n",
    "    t_ = re.sub('- ','',t_)\n",
    "    #url_reg  = r'[a-z]*[:.]+\\S+'\n",
    "    #t_ = re.sub(url_reg, '', t_)\n",
    "    t_ = re.sub('([.,!?()])', r' \\1 ', t_)\n",
    "    t_ = re.sub('\\\"', ' \\\" ',t_)\n",
    "    t_ = re.sub('$', ' $ ',t_)\n",
    "    t_ = re.sub(r'\\'s', ' \\'s', t_)\n",
    "    t_ = re.sub(r'\\'re', ' \\'re', t_)\n",
    "    t_ = re.sub(r'\\'ll', ' \\'ll', t_)\n",
    "    t_ = re.sub(r'\\'m', ' \\'m', t_)\n",
    "    t_ = re.sub(r'\\'d', ' \\'d', t_)\n",
    "    t_ = re.sub(r'can\\'t', 'can n\\'t', t_)\n",
    "    t_ = re.sub(r'n\\'t', ' n\\'t', t_)\n",
    "    t_ = re.sub(r'sn\\'t', 's n\\'t', t_)\n",
    "    t_ = re.sub('\\s{2,}', ' ', t_)\n",
    "    t_ = t_.lower()\n",
    "    mydict = us_gb_dict()\n",
    "    t_ = replace_all(t_, mydict)\n",
    "    return(t_)\n",
    "\n",
    "def clean_par(t_):\n",
    "    #t_ = re.sub('\\s+',' ',t_)\n",
    "    t_ = re.sub('- ','',t_)\n",
    "    #url_reg  = r'[a-z]*[:.]+\\S+'\n",
    "    #t_ = re.sub(url_reg, '', t_)\n",
    "    t_ = re.sub('([.,!?()])', r' \\1 ', t_)\n",
    "    t_ = re.sub('\\\"', ' \\\" ',t_)\n",
    "    t_ = re.sub('$', ' $ ',t_)\n",
    "    t_ = re.sub(r'\\'s', ' \\'s', t_)\n",
    "    t_ = re.sub(r'\\'re', ' \\'re', t_)\n",
    "    t_ = re.sub(r'\\'ll', ' \\'ll', t_)\n",
    "    t_ = re.sub(r'\\'m', ' \\'m', t_)\n",
    "    t_ = re.sub(r'\\'d', ' \\'d', t_)\n",
    "    t_ = re.sub(r'can\\'t', 'can n\\'t', t_)\n",
    "    t_ = re.sub(r'n\\'t', ' n\\'t', t_)\n",
    "    t_ = re.sub(r'sn\\'t', 's n\\'t', t_)\n",
    "    #t_ = re.sub('\\s{2,}', ' ', t_)\n",
    "    t_ = t_.lower()\n",
    "    mydict = us_gb_dict()\n",
    "    t_ = replace_all(t_, mydict)\n",
    "    return(t_)\n",
    "\n",
    "\n",
    "def us_gb_dict():    \n",
    "    filepath = 'us_gb.txt'\n",
    "    with open(filepath, 'r') as fp:  \n",
    "        read = fp.read()\n",
    "    us = []\n",
    "    gb = []\n",
    "    gb_f = True\n",
    "\n",
    "    for i in read.splitlines():\n",
    "        line = i.strip()\n",
    "        #print(line)\n",
    "        if line == \"US\":\n",
    "            gb_f = False      \n",
    "        elif gb_f == True:\n",
    "            gb.append(line)\n",
    "        else:\n",
    "            us.append(line)\n",
    "    us2gb = dict(zip(gb, us))\n",
    "    return us2gb\n",
    "\n",
    "\n",
    "def replace_all(text, mydict):    \n",
    "    for gb, us in mydict.items():\n",
    "        text = text.replace(gb, us)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(df, col = 'essay'):\n",
    "    t = []\n",
    "    t_par = []\n",
    "    for i in df[col]:\n",
    "        t.append(clean(i))\n",
    "        t_par.append(clean_par(i))\n",
    "    df['text1'] = t\n",
    "    df['text_par'] = t_par\n",
    "    df['label'] = df['domain1_score']\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data(path, sets, path_o, folders = ['fold_0', 'fold_1','fold_2','fold_3', 'fold_4']):\n",
    "    \n",
    "    for i in range(len(folders)):\n",
    "        f_p = os.path.join(path,folders[i])\n",
    "        f_path_o = os.path.join(path_o,folders[i])\n",
    "        \n",
    "        test_str = os.path.join(f_p,'test.tsv')\n",
    "        train_str = os.path.join(f_p,'train.tsv')\n",
    "        dev_str = os.path.join(f_p,'dev.tsv')\n",
    "\n",
    "        df_test = pd.read_csv(test_str, sep = '\\t')\n",
    "        df_train = pd.read_csv(train_str, sep = '\\t')\n",
    "        df_dev = pd.read_csv(dev_str, sep = '\\t')\n",
    "\n",
    "        df_test = clean_text(df_test)\n",
    "        df_train = clean_text(df_train)\n",
    "        df_dev = clean_text(df_dev)\n",
    "\n",
    "        for i in sets:\n",
    "            path_i = os.path.join(f_path_o,str(i))\n",
    "            if not os.path.exists(f_path_o):\n",
    "                os.makedirs(f_path_o)\n",
    "\n",
    "            if not os.path.exists(path_i):\n",
    "                os.makedirs(path_i)\n",
    "                \n",
    "            df_test[df_test['essay_set']==i].to_csv(os.path.join(path_i,'test.csv'))\n",
    "            df_train[df_train['essay_set']==i].to_csv(os.path.join(path_i,'train.csv'))\n",
    "            df_dev[df_dev['essay_set']==i].to_csv(os.path.join(path_i,'dev.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data(path,sets,directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
